import com.aliasi.symbol.MapSymbolTable;
import com.aliasi.tokenizer.EnglishStopTokenizerFactory;
import com.aliasi.tokenizer.IndoEuropeanTokenizerFactory;
import com.aliasi.tokenizer.NGramTokenizerFactory;
import com.aliasi.tokenizer.TokenFeatureExtractor;
import com.aliasi.tokenizer.TokenNGramTokenizerFactory;
import com.aliasi.tokenizer.Tokenizer;
import com.aliasi.util.BoundedPriorityQueue;
import com.aliasi.util.Pair;
import com.aliasi.util.Proximity;
import com.aliasi.util.ScoredObject;
import com.aliasi.util.Streams;
import com.aliasi.classify.*;
import com.aliasi.features.BoundedFeatureExtractor;
import com.aliasi.lm.LanguageModel.Dynamic;
import com.aliasi.lm.TokenizedLM;
import com.aliasi.matrix.CosineDistance;
import com.aliasi.matrix.Vector;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.MinMaxPriorityQueue;
import java.io.*;
import java.math.BigDecimal;
import java.util.*;
import edu.jhu.htm.core.*;

/**                 DOCUMENT GEOCODER
 * 
 * This function receives the following Strings:
 * 
 * * 1 : Train data File name
 * * 2 : Coordinates File name
 * * 3 : Test data File name
 * * 4 : Resolution of the HTM grid
 * * 5 : 1 if you want to create token-based language model classifiers, 
 *       0 if you want to create character-based language models
 * * 6 : Number of n-grams to use 
 * * 7 : Resolution for the first level classifier
 * * 8 : 0 for training and evaluate the model
 *       1 for training the model only
 *       2 for evaluating the model only
 *       3 for classifying a collection
 * * 9 : collection file name if you want to classify a collection
 * 
 * File Format :
 * 
 * * Train data and test data have the following format in a .txt file :
 * * * Title\tText 
 * * Coordinates data has the following format in a .txt file :
 * * * Title\tlatitude\tlongitude 
 *       
 * * Requirements:
 * 
 * lingpipe package available at http://alias-i.com/lingpipe/
 * guava-11.0.1 package available at http://code.google.com/p/guava-libraries/ 
 * 
 * Example:
 * * java DocumentGeocoder train_data.txt coords.txt test_data.txt 4 0 8 0 0
 * 
 *
 */
public class DocumentGeocoderLMNew{

    File trainData;
    File coorData;
    File testData;
    
    HashMap<String, String[]> coorHash = new HashMap<String, String[]>();
    HashMap<String, String> textHash = new HashMap<String, String>();
    HashMap<String, String> testTextHash = new HashMap<String, String>();
    File fileAnalysis = new File("analysis.txt");
    FileWriter analysisWriter = new FileWriter(fileAnalysis);
    String[] mCategories;
    String[] mCategoriesInit;
    JointClassifier mClassifier;

    int nGram ;
    int resolution;					//Resolução da grelha
    HTMindexImp spindex; 
    HTMindexImp spindexInit;
    int tokenized;
    int removed;
    int multi;
    HashMultimap<String, String> catClassifications;
    HashMultimap<String, String> catTestClassifications;
    int resInit;
    String classifierDir = "Classifiers";
   
    DocumentGeocoderLMNew(String[] args) throws IOException {
    	removed = 0;
        System.out.println("\nDocument Geocoder");
        catClassifications = catClassifications.create();
        catTestClassifications = catTestClassifications.create();
        trainData = new File(args[0]);
        coorData = new File(args[1]);
        testData = new File(args[2]);	
        resolution =  Integer.parseInt(args[3]);	
        spindex = new HTMindexImp(resolution); 
        tokenized = Integer.parseInt(args[4]);	
        nGram = Integer.parseInt(args[5]);
        resInit = Integer.parseInt(args[6]); 
        multi = resolution;
        multi -= resInit + 1;
        spindexInit = new HTMindexImp(resInit);
        coorHash = getCoorHash(coorData);
        
        if(args[7].equals("2")){	
        	
        	addCoorHash(new File("travelCoords.txt"));
        }
        textHash = getTextHash(trainData);
        
        mCategoriesInit = getCatList(coorData);
        
        
        
	    mCategories = resolutionMultiAdapter(multi);  
	    System.out.println("Total of " + mCategories.length + " in second resolution");
	    if ( tokenized == 1 ){ 
	    	 mClassifier = CalibratedLMClassifier.createTokenized(mCategoriesInit, new
	    		IndoEuropeanTokenizerFactory(), nGram);}    
	    	else {
	    		mClassifier = CalibratedLMClassifier.createNGramProcess(mCategoriesInit,nGram);
	    	}
	    
	    System.out.println("\nCoordinate Data=" + coorData);
	    System.out.println("\nText Data 1=" + trainData);
	    
	        
    }
     
    // add coordinates to the coordinates hash
    void addCoorHash(File coordfile)throws IOException{
    	String line = null;    
    	String[] coords = new String[2]; 
    	
    	BufferedReader input = new BufferedReader(new FileReader(coordfile));
    	while ( ( line = input.readLine() ) != null ) {
    		System.out.println("adding : "+ line);
    		coords = new String[2];
    	    String fields[] = line.split("\t");
    	    if ( fields.length != 3) continue;
    	    coords[0]= fields[1];
    	    coords[1]= fields[2];
    	    coorHash.put(fields[0], coords);
    	}
    	
    	
    }
    
    //Gets a HashMap containing title -> {lat, lon} from a given coordinates data file 
    
    HashMap<String, String[]> getCoorHash(File coordfile) throws IOException{
    	String line = null;    
    	String[] coords = new String[2]; 
    	HashMap<String, String[]> hash = new HashMap<String, String[]>();
    	BufferedReader input = new BufferedReader(new FileReader(coordfile));
    	while ( ( line = input.readLine() ) != null ) {
    		
    		coords = new String[2];
    	    String fields[] = line.split("\t");
    	    if ( fields.length != 3) continue;
    	    coords[0]= fields[1];
    	    coords[1]= fields[2];
    	    hash.put(fields[0], coords);
    	}
    	return hash;
    }
    
    
    //Gets an Hashmap from a given test or train data file
    
    HashMap<String, String> getTextHash(File textFile) throws IOException{
    	String line = null;    
    	HashMap<String, String> hash = new HashMap<String, String>();
    	BufferedReader input = new BufferedReader(new FileReader(textFile));
    	while ( ( line = input.readLine() ) != null ) {
    	    String fields[] = line.split("\t");
    	    if ( fields.length != 2) continue;
    	    
    	    hash.put(fields[0], fields[1]);
    	}
    	return hash;
    
    }
    
    // Adapts the initial category from the second level category, to the reduced category
    String catAdaptor(String category){
    	for(String cat : mCategories){
   		 if(cat.equals(category))
   			 return category;
   		 
   		 if(category.contains(cat))
   			 return cat;
    	}
    	return category;
    }
    
    
    
    // Returns the category for the given coordinates at a given resolution
    String getCat ( Double lat , Double lon, HTMindexImp spind )  {
        if ( lat == 0.0 ) lat = 0.25;
        if ( lat == -90.0 ) lat = -89.75;
     if ( lat == 90.0 ) lat = 89.75;
        if ( lon == 0.0 ) lon = 0.25;
        if ( lon == -180.0 ) lon = -179.75;
     if ( lon == 180.0 ) lon = 179.75;
     try { 
    	 String category = spind.lookup( lon , lat );     
    	 for(String cat : mCategories){
    		 if(cat.equals(category))
    			 return category;
    		 
    		 if(category.contains(cat))
    			 return cat;
    		
    	 }	     		 
    
        return category;
     } catch ( Exception ex ) { return null; } 
       }
    
    
    
    
    
    String getCatInit ( Double lat , Double lon, HTMindexImp spind)  {
        if ( lat == 0.0 ) lat = 0.25;
        if ( lat == -90.0 ) lat = -89.75;
     if ( lat == 90.0 ) lat = 89.75;
        if ( lon == 0.0 ) lon = 0.25;
        if ( lon == -180.0 ) lon = -179.75;
     if ( lon == 180.0 ) lon = 179.75;
     try { 
    	     	 
    	 return spind.lookup( lon , lat );     	     		 
    	
     } catch ( Exception ex ) { return null; } 
       }
    
    
    
    
    
    //Returns a vector of neighbouring categories of the given category
    String[] getNeighbourCats ( String cat ) {
	try { 
		
		Object v[] = HTMfunc.nameToTriangle(cat); 
		Domain d = new Domain();
                Vector3d v1 = new Vector3d( ((double[])(v[0]))[0] , ((double[])(v[0]))[1], ((double[])(v[0]))[2] );
                Vector3d v2 = new Vector3d( ((double[])(v[1]))[0] , ((double[])(v[1]))[1], ((double[])(v[1]))[2] );
                Vector3d v3 = new Vector3d( ((double[])(v[2]))[0] , ((double[])(v[2]))[1], ((double[])(v[2]))[2] );
		Convex c = new Convex(v1,v2,v3);
		d.add(c);
		int n = spindex.intersect(d).toString(true).split(" ").length;
		
		Set<String> aux = new HashSet<String>();
		  for (String s : spindex.intersect(d).toString(true).split("[ \t\n]") ){
			  aux.add(s);			  
		  }
		  
		  return aux.toArray(new String[0]);
		  
	} catch ( Exception ex ) { ex.printStackTrace(); return null; } 
    }
    
    
    
    
    
    Double getCatLatitude ( String cat ) {
   	 try { 
   	    return spindex.idToPoint(cat).dec();
   	    } catch ( Exception ex ) { return null; }
    }
    
    
    
    
    
    
    Double getCatLongitude ( String cat ) {
    	try {
    	double aux = spindex.idToPoint(cat).ra() ;
    	if ( aux > 180.0 ) aux = -360.0 + aux;
    	return aux;
    	} catch ( Exception ex ) { return null; }
    }

    
    
    //returns the area of a certain category
     double getArea ( String cat ) {
    	 try { 
    		 
    	  Object v[] = HTMfunc.nameToTriangle(cat); 
    	  Domain d = new Domain();
    	        Vector3d v1 = new Vector3d( ((double[])(v[0]))[0] , ((double[])(v[0]))[1], ((double[])(v[0]))[2] );
    	        Vector3d v2 = new Vector3d( ((double[])(v[1]))[0] , ((double[])(v[1]))[1], ((double[])(v[1]))[2] );
    	        Vector3d v3 = new Vector3d( ((double[])(v[2]))[0] , ((double[])(v[2]))[1], ((double[])(v[2]))[2] );
    	  double v1lat = v1.dec();
    	  double v2lat = v2.dec();
    	  double v3lat = v3.dec();
    	  double v1lon = v1.ra();
    	  double v2lon = v2.ra();
    	  double v3lon = v3.ra();
    	  if(v1.ra() > 180.0) v1lon =  -360 + v1.ra();
    	  if(v2.ra() > 180.0) v2lon =  -360 + v2.ra();
    	  if(v3.ra() > 180.0) v3lon =  -360 + v3.ra();
    	  
    	  double l1 = VincentyDistanceCalculator.getDistance(v1lat ,v1lon , v2lat , v2lon ); 
    	  double l2 = VincentyDistanceCalculator.getDistance(v2lat ,v2lon , v3lat , v3lon );
    	  double l3 = VincentyDistanceCalculator.getDistance(v3lat ,v3lon , v1lat , v1lon );
    	  double p = (l1 + l2 + l3) / 2.0;
    	  double area = Math.sqrt( p * (p-l1) * (p-l2) * (p-l3) );
    	  return area;
    	 } catch ( Exception ex ) { ex.printStackTrace(); return -1; } 
   }
    
    
    
    
    

    // Average of latitude according to probability of categories  
    Double getLatitude ( Map<String,Double> cats ) {
        List<Double> lats = new ArrayList<Double>();
        List<Double> latProbs = new ArrayList<Double>();
        
        //para cada categoria
        for ( String cat : cats.keySet() ) {
        	//adiciona a lista a latitude dessa categoria
        	lats.add(getCatLatitude(cat));
        	//adiciona a lista a probabilidade dessa categoria
        	latProbs.add(cats.get(cat));
        }
        Double val = 0.0;
        Double sum = 0.0;
        
        
        for ( int i = 0 ; i < lats.size() ; i++ ) {
                val += lats.get(i) * latProbs.get(i);
                sum += latProbs.get(i);
        }
        return val / sum;
    }
    
    
    
    
   Double getLongitude ( Map<String,Double> cats ) {
    List<Double> lons = new ArrayList<Double>();
    List<Double> lonProbs = new ArrayList<Double>();
	for ( String cat : cats.keySet() ){ 
		lons.add(getCatLongitude(cat)); 
		lonProbs.add(cats.get(cat));		
	}
	Double val = 0.0;
	Double sum = 0.0;
	for ( int i = 0 ; i < lons.size() ; i++ ) {
        val += lons.get(i) * lonProbs.get(i);
        sum += lonProbs.get(i);
	}
	
	return val / sum;
    }
    
   
   
   
   
   
   
    //Writes a file with an analysis of the number of categories and the average area
    void catAnalysis (File data) throws IOException {
   	Set<String> cats = new HashSet<String>();
   	BufferedReader input = new BufferedReader(new FileReader(data));
   	String line = null;
   	double area = 0.0;
   	int num = 0;
   	while ( ( line = input.readLine() ) != null ) {
   		String fields[] = line.split("\t");
   		if ( fields.length != 3) continue;
   		String category = getCatInit( new Double(fields[1]) , new Double(fields[2]), spindex);
   		cats.add(category);
   		num++;
   		area += getArea(category);
   		}
   	for ( String cat : cats ) System.out.println("Category : " + cat);
   	System.out.println("Total of " + cats.size() +" categories in the training data");
   	System.out.println("Average area of categories : " +  (double) (area/(double)num));
   	
   	analysisWriter.write("Total of " + cats.size() +" categories in the training data \n");
   	analysisWriter.write("Average area of categories : " +  (double) (area/(double)num) + '\n');
   	analysisWriter.flush();
    }
   
    
    
    String[] getCatList (File data) throws IOException {
    	Set<String> cats = new HashSet<String>();
    	BufferedReader input = new BufferedReader(new FileReader(data));
    	String line = null;
    	while ( ( line = input.readLine() ) != null ) {
    		String fields[] = line.split("\t");
    		if ( fields.length != 3) continue;
    		String category = getCatInit( new Double(fields[1]) , new Double(fields[2]), spindexInit);
    		cats.add(category);
    		}
    	for ( String cat : cats ) System.out.println("Category : " + cat);
    	System.out.println("Total of " + cats.size() +" categories in the training data");
    	return cats.toArray(new String[0]);
    }
    
    
    
    
    
    


    void run() throws ClassNotFoundException, IOException {    	
    	train();
        train2();
        evaluate();
        analysisWriter.close();
        

    }
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    
    /*
     * Adapts the resolution of the categories
     * If the neighbours of a category do not have any assigned training documents
     * Collapse the category into a trixel of the next smaller resolution
     * Repeats recursively until possible
     * 
     */ 
    String[] resolutionMultiAdapter(int resulotionMax) throws IOException{
	    BufferedReader inputText = new BufferedReader(new FileReader(trainData));
        String line = null;
	    HashMap<String, Integer> numClass = new HashMap<String, Integer>();
	    HashMap<String, Integer> numClassAux = new HashMap<String, Integer>();
	    HashMap<String, String> textHash = new HashMap<String, String>();
       while ( ( line = inputText.readLine() ) != null ) {       
	    String fields[] = line.split("\t");
	    
	    if ( fields.length != 2) continue;

	        String category = getCatInit( new Double(coorHash.get(fields[0])[0]) , new Double(coorHash.get(fields[0])[1]), spindex);
	        
	        textHash.put(fields[0], fields[1]);
            if(numClass.get(category) == null){

            	numClass.put(category, 1);
            	numClassAux.put(category, 1);
           }
           else {
        	   numClass.put(category, numClass.get(category) + 1 );
        	   numClassAux.put(category, numClassAux.get(category) + 1);
        	   
           }
          
       }
       
       for(String aux : numClass.keySet()){
    	   
    	    HashSet<String> classeReduzivel = new HashSet<String>();
    	    HashSet<String> vizinhosRemAux = new HashSet<String>();
    	    int maxI = 0;
       		boolean reduzivel = true; //flag para assinalar se os vizinhos têm documentos
       		int i ;
       		for(i = 1; i <= resulotionMax ; i ++){
       			for(String aux2 : numClass.keySet()){
       				if(aux2.startsWith(aux.substring(0, aux.length() - i )) && aux2 != aux){
       					reduzivel = false;      					 						
       				}
       			}
       			if(reduzivel == true){
       				maxI = i;
       				numClassAux.remove(aux);
       			}
       			else{
       				break;
       				}
       			}
       		if(maxI > 0)
       			numClassAux.put(aux.substring(0, aux.length()-maxI), 1);
       			
       		}
       
       numClass.clear();       
       textHash.clear();
       
         
       
   return (String[]) numClassAux.keySet().toArray(new String[numClassAux.size()]);
   }
   
   
    
    
    
    
    
    
    
    
    
    
   // Statistical analysis of a given collection 
   void wordCount(String dados) throws IOException{
	   HashMap<String, String> hash = new HashMap<String, String>();
	   ArrayList<Integer> list = new ArrayList<Integer>();
	   if(dados.equals("train"))
		   hash = textHash;
	   else if(dados.equals("test"))
		   hash = getTextHash(testData);
	   else {
		   System.out.println("oooops"); 
		   System.exit(0);
	   }
	   int numWords = 0;
	   for(String title : hash.keySet()){
		   int num = 0;
		   char[] aux;
		   aux = hash.get(title).toCharArray();
		   
		   Tokenizer token =   new IndoEuropeanTokenizerFactory().tokenizer(hash.get(title).toCharArray(), 0, hash.get(title).length());
		   num = token.tokenize().length;
		   //System.out.println(num);
		   numWords += num;
		   list.add(num);
		   
	   }
	   System.out.println("numero de palavras no " + dados +" :" + numWords); 
	   System.out.println("numero medio de palavras por documento " + dados +" :" + numWords/hash.size()); 
	   System.out.println("desvio padrao de palavras do " + dados + " : " + deviation(list, numWords/hash.size()));
	   try {
		analysisWriter.write("numero de palavras no " + dados +" :" + numWords + '\n');
		analysisWriter.write("numero medio de palavras por documento " + dados +" :" + numWords/hash.size() + '\n'); 
		analysisWriter.write("desvio padrao de palavras do " + dados + " : " + deviation(list, numWords/hash.size()) + '\n');
		analysisWriter.flush();
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	   
   }
    

   
   
   
   /*
    * 
    * Train the first level classifier,
    * and writes it into a file in dir "Classifiers"
    * 
    * 
    * 
    */
   
   void train() throws IOException, ClassNotFoundException {
        int numTrainingCases = 0;
        int numTrainingChars = 0;
        System.out.print("\nTraining.");
	    int num = 0;
	    
	    for(String title: textHash.keySet()){
	       
        	System.out.print(".");
        	if (num++ % 1000 == 0) System.out.println(" " + num + " ");
        	
        	String category = getCatInit( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindexInit);
            String text = title + " " + textHash.get(title);
            
            Classification classification = new Classification(category);
            ++numTrainingCases;
            numTrainingChars += text.length();
            Classified<CharSequence> classified = new Classified<CharSequence>(text,classification);
            ((CalibratedLMClassifier)mClassifier).handle(classified);
            catClassifications.put(category, title);
            
            	
        }
	        
        System.out.println("  # Training Cases=" + numTrainingCases);
        System.out.println("  # Training Chars=" + numTrainingChars);
       
        if(!new File(classifierDir).exists())
        	new File(classifierDir).mkdir();

        File auxf = new File(classifierDir+"/initClassifier_" + resolution + '_' + nGram + '_' + tokenized);
        FileOutputStream fileOut = new FileOutputStream(auxf);
        BufferedOutputStream bufOut = new BufferedOutputStream(fileOut);
        ObjectOutputStream objOut = new ObjectOutputStream(bufOut);
        ((CalibratedLMClassifier)mClassifier).compileTo(objOut);
        Streams.closeQuietly(objOut);
        Streams.closeQuietly(bufOut);
        Streams.closeQuietly(fileOut);
        FileInputStream fileIn = new FileInputStream(auxf);
        BufferedInputStream bufIn = new BufferedInputStream(fileIn);
        ObjectInputStream objIn = new ObjectInputStream(bufIn);
        mClassifier= (JointClassifier) (objIn.readObject());
        Streams.closeQuietly(objIn);
        Streams.closeQuietly(bufIn);
        Streams.closeQuietly(fileIn);
        
      
    }
   
   
   
	   
	   
	   
	   
	   
	   
  
   
   void bestCharAnalysis(FileWriter writer, String word, HashSet<String> subCats) throws IOException{
	   MinMaxPriorityQueue<Pair<Double,String>> w1Queue; 

       w1Queue = MinMaxPriorityQueue.orderedBy(new MaxComparator()).maximumSize(10).create();  
       
	   
	   for(String catAux : subCats){
	    	try{
		    Dynamic d = ((CalibratedLMClassifier) mClassifier).languageModel(catAux);
	    	w1Queue.add(new Pair<Double,String>(Math.pow(2, d.log2Estimate(word)), catAux));
	    	}catch(ClassCastException e){e.printStackTrace();}
	    	
	   }
	    
	   //Char analysis
	    
       Pair<Double, String> w1;
	    
	   w1 = w1Queue.poll();
	    
	    		
	    while(w1 != null){
       	
       	writer.write(""+ word +'\t' + w1.a() + '\t' + w1.b() + '\t' + getCatLatitude(w1.b()) + '\t' + getCatLongitude(w1.b()) +'\n');
	    	w1 = w1Queue.poll();
       }
       
   }
   
   
   
   
   
   
   
   
   
   
   
   
   
   //ACHO QUE TA....
   HashSet<String> catsRegion(Pair<Double,Double> bCoords1 ,  Pair<Double,Double> bCoords2){

	   double distanceLat =  Math.sqrt(Math.pow(bCoords1.a() - bCoords2.a(), 2));
	   double distanceLon =  Math.sqrt(Math.pow(bCoords1.b() - bCoords2.b(), 2));
	   double divisionsLat = distanceLat / 10;
	   double divisionsLon = distanceLon / 10;
	   double minLat = Math.min(bCoords1.a(), bCoords2.a());
	   double minLon = Math.min(bCoords1.b(), bCoords2.b());
	   double maxLat = Math.max(bCoords1.a(), bCoords2.a());
	   double maxLon =  Math.max(bCoords1.b(), bCoords2.b());
	   double avArea = 0.0;
	   double lado = VincentyDistanceCalculator.getDistance(minLat, minLon, maxLat, minLon);
	   double altura = VincentyDistanceCalculator.getDistance(minLat, minLon, minLat, maxLon);
	   double fraccaoLat= VincentyDistanceCalculator.getDistance(minLat, minLon, minLat + divisionsLat, minLon);
	   double fraccaoLon = VincentyDistanceCalculator.getDistance(minLat, minLon, minLat, minLon + divisionsLon);
	   avArea += getArea(getCat(minLat,minLon,spindex));
	   avArea += getArea(getCat(minLat,maxLon,spindex));
	   avArea += getArea(getCat(maxLat,minLon,spindex));
	   avArea += getArea(getCat(maxLat,maxLon,spindex));
	   avArea /= 4;
	   System.out.println("lado " + lado + " altura " + altura + " area " + lado*altura + " av area trixels " +avArea + " frac lad " + fraccaoLat + " frac lon " + fraccaoLon);
	   
	   
	   HashSet<String> catHash = new HashSet<String>();
	   for(double b = minLon; b<maxLon ; b += divisionsLon){
		   
		   for(double a = minLat; a<maxLat; a+= divisionsLat){
			   catHash.add(getCat(a , b , spindex));      
			  
		   }
	   }
	   return catHash;
   }
   
   
   
   
   
   /*
    * Second level 
    * For each category present in the first level
    * create a second classifier with a higher resolution
    * each classifier is written in a file in the dir Classifiers
    * 
    */
   
   
   void train2()throws IOException, ClassNotFoundException {
	   
       System.out.print("\nTraining.");
       wordCount("train");
       
       
	   
       for(String cat : catClassifications.keySet()){
            
		   int num = 0;
		   int numTrainingCases = 0;
	       int numTrainingChars = 0;
		   HashSet<String> cats = new HashSet<String>();
		   
		   
		   for(String title : catClassifications.get(cat)){
			   cats.add(getCat( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindex));
		   }
		   
	       HashSet<String> subCats = new HashSet<String>(); // hashSet com todas as categorias do treino
		   
	       
	       if(tokenized == 1){
	    	   try{
	    	   mClassifier = CalibratedLMClassifier.createTokenized(cats.toArray(new String[0]), new IndoEuropeanTokenizerFactory(), nGram);
	    	   }catch(IllegalArgumentException i){continue;}
	       }else{
	    	   try{
	    		   mClassifier = CalibratedLMClassifier.createNGramProcess(cats.toArray(new String[0]),nGram);
	    	   }catch(IllegalArgumentException i){continue;}
	       
	       }	
		   for(String title : catClassifications.get(cat)){
		        
	        	System.out.print(".");
	        	if (num++ % 1000 == 0) 
	        		System.out.print(" " + num + " ");
	        	
	        	String category = getCat( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindex);
	            subCats.add(category);
	            String text = title + " " + textHash.get(title);
	            
	            Classification classification = new Classification(category);
	            ++numTrainingCases;
	            numTrainingChars += text.length();
	            Classified<CharSequence> classified = new Classified<CharSequence>(text,classification);
	            ((CalibratedLMClassifier ) mClassifier).handle(classified);
	            
	            
	        }
		    
		    System.out.println("  # Training Cases for Category " + cat + " :" + numTrainingCases);
	        System.out.println("  # Training Chars=" + numTrainingChars);
	        File auxf = new File(classifierDir+'/'+cat + '_' + resolution + '_' + resInit + '_' + nGram + '_' + tokenized + "_train" );
	        FileOutputStream fileOut = new FileOutputStream(auxf);
	        BufferedOutputStream bufOut = new BufferedOutputStream(fileOut);
	        ObjectOutputStream objOut = new ObjectOutputStream(bufOut);
	        ((CalibratedLMClassifier) mClassifier).compileTo(objOut);
	        Streams.closeQuietly(objOut);
	        Streams.closeQuietly(bufOut);
	        Streams.closeQuietly(fileOut);
	        FileInputStream fileIn = new FileInputStream(auxf);
	        BufferedInputStream bufIn = new BufferedInputStream(fileIn);
	        ObjectInputStream objIn = new ObjectInputStream(bufIn);
	        mClassifier= (JointClassifier) objIn.readObject();
	        Streams.closeQuietly(objIn);
	        Streams.closeQuietly(bufIn);
	        Streams.closeQuietly(fileIn);
	        
	        
	   }
	
	   catClassifications.clear();
	   
   }

   
   
   
   private HashMap<String, Double> sortHashMap(HashMap<String, Double> input){
	    Map<String, Double> tempMap = new HashMap<String, Double>();
	    for (String wsState : input.keySet()){
	        tempMap.put(wsState,input.get(wsState));
	    }

	    List<String> mapKeys = new ArrayList<String>(tempMap.keySet());
	    List<Double> mapValues = new ArrayList<Double>(tempMap.values());
	    HashMap<String, Double> sortedMap = new LinkedHashMap<String, Double>();
	    TreeSet<Double> sortedSet = new TreeSet<Double>(mapValues);
	    Object[] sortedArray = sortedSet.toArray();
	    int size = sortedArray.length;
	    for (int i=0; i<size; i++){
	        sortedMap.put(mapKeys.get(mapValues.indexOf(sortedArray[i])), 
	                      (Double)sortedArray[i]);
	    }
	    return sortedMap;
	}
   
   
   
   
   
   
   // Assembles new Terms in Test vs Training
   HashMap<String,Double> regionAnalysis(TokenizedLM backgroundModel, String cat, HashMap<String,Double> scoreHash){
		
	   System.out.println("\nAssembling New Terms in Test vs. Training");
	   SortedSet<ScoredObject<String[]>> newTerms 
	       =  ( (TokenizedLM) (((CalibratedLMClassifier) mClassifier).languageModel(cat))).newTermSet(2, 3, 5, backgroundModel);
	   System.out.println("\nNew Terms in Order of Signficance:");
	   
	   for (ScoredObject<String[]> nGram : newTerms) {
           double score = nGram.score();
           String[] toks = nGram.getObject();
           
           String accum = "";
           for (int j=0; j<toks.length; ++j) {
               accum += " "+toks[j];
           }
           
           
           if(scoreHash.containsKey(accum))
        	   scoreHash.put(accum, scoreHash.get(accum) + score);
           else scoreHash.put(accum, score);
           System.out.println("Score: "+ score +" with :"+accum);
       }
	   return scoreHash;
       
}
   
   

   
   
/*
 * Analysis train data, and searches documents that have words for analysis, writes a file with:
 * * word\tlatitude\tlongitude
 * 
 *    
 */
void wordAnalysis(String lingua, String[] palavras) throws IOException{
	   
	   String line = null;    
	   	String[] coords = new String[2]; 
	   	HashMap<String, String[]> hash = new HashMap<String, String[]>();
	   	BufferedReader reader = new BufferedReader(new FileReader(trainData));
	   	int size = palavras.length;
	   	FileWriter w = new FileWriter(new File("palavrasMapa" + lingua + ".txt"));
	   	System.out.println("analisar palavras:");
	   
	   	for(int i = 0; i<size; i++)
	   		System.out.println(palavras[i]);
	   	

	   	w.write("tipo\tLatE\tLonE\n");
	   	 //percorre cada linha
	    while ( ( line = reader.readLine() ) != null ) {
	  
	    	
	   	    String fields[] = line.split("\t");
	   	    String frase[] = fields[1].split("[ .:,!?\n]");
	   	    
	   	    
	   	    for(int i= 0; i< size; i++){
	   	    	String palavra = palavras[i];
	   	    
	   	    	for(int j = 0; j<frase.length ; j++){
	   	    		String token = frase[j];
	   	    		
	   	    		if(token.equals(palavra)|| token.toLowerCase().equals(palavra) ){
	   	    			w.write(palavras[i] + "\t" + coorHash.get(fields[0])[0] + '\t' + coorHash.get(fields[0])[1] + '\n');
	   		   	    	break;
	   	    		}
	   	    	}
	   	    }
	   	      
	   }
	  w.close();
   }
   
   
   
   
   
   double deviation(ArrayList<Integer> data, int mean){
	   
	      int n = data.size();	     
	      
	      // calculate the sum of squares
	      int sum = 0;
	      for ( int i=0; i<n; i++ ){
	    	 
	         final int v = data.get(i) - mean;
	         //System.out.println(v);
	         sum += Math.pow(v, 2);
	         }
	      System.out.println(sum);
	      return Math.sqrt( (double)((double)sum / (double) n) );
	   
}
    
   double deviation(List<Double> data, double mean){
	   
	      
	      int n = data.size();	     
	      
	      // calculate the sum of squares
	      double sum = 0;
	      for ( int i=0; i<n; i++ ){
	         final double v = data.get(i) - mean;
	         sum += v * v;
	         }
	      
	      return Math.sqrt( sum / n );
   }
    
   
   
   
   private Pair<Double,Double> adjustCoords(ArrayList<Pair<Double, Double>> coordsL, ArrayList<Double > scoreL, Pair<Double, Double> realCoords){
	   double lat = realCoords.a().doubleValue();
	   double lon = realCoords.b().doubleValue();
	   double prob = 1.0;
	   for(int i = 0; i<coordsL.size(); i++){
		   lat += coordsL.get(i).a().doubleValue() * scoreL.get(i);
		   lon += coordsL.get(i).b().doubleValue() * scoreL.get(i);	
		   prob+= scoreL.get(i);
	   }
	   
	   return new Pair<Double,Double>(new Double(lat/prob), new Double(lon/prob));
	        
   }
   
   
   
   
   
   
   

   KnnNeighbours trainSimilarityModel(){
	   
	   String testText = new String();
       String testCategory = new String();

       
	   EnglishStopTokenizerFactory english = new EnglishStopTokenizerFactory(new IndoEuropeanTokenizerFactory()); // baselibe
       TokenNGramTokenizerFactory tngrams = new TokenNGramTokenizerFactory(english, 1, 2); 
       
       TokenFeatureExtractor tokenExtractor = new TokenFeatureExtractor(tngrams);
       Proximity<Vector> distanceFunction;
       distanceFunction = new CosineDistance();
       BoundedFeatureExtractor extractor = new BoundedFeatureExtractor(tokenExtractor , 5,  Double.POSITIVE_INFINITY);
       KnnNeighbours simClassifier = new KnnNeighbours(extractor, 5, distanceFunction, true);	
	  
       System.out.println("Training similarity model.");
       for(String title : textHash.keySet()){

       	 testText = textHash.get(title); 
       	 testCategory =  getCat( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindex);
       	 Classification classificationSim = new Classification( getCat( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindex));
         Classified<CharSequence> classified = new Classified<CharSequence>(textHash.get(title),classificationSim);         	
         simClassifier.handle(classified, new Double(coorHash.get(title)[0]), new Double(coorHash.get(title)[1]));     

       }
	   
       
	   return simClassifier;
	   
   }
   
   
   /*
    * 
    * Classifies a given collection with a pair of coordinates latitude/longitude
    * Writes file with:
    * * title	latitude	longitude	text
    * Returns written file
    */
   
   
   File classifyCollection(String file) throws IOException, ClassNotFoundException{
	   System.out.println("Classifying collection.");
	   int num = 0;
	   File collectionFile = new File(file + ".txt");
	   BufferedReader collection = new BufferedReader(new FileReader(collectionFile));
	   
	   String line = null;
	   
	   File classifiedGeo = new File(file + "_classifiedGeo.txt");
	   FileWriter geoWritter = new FileWriter(classifiedGeo);
	   FileInputStream fileIn = new FileInputStream(classifierDir+"/initClassifier_" + resolution + '_' + nGram + '_' + tokenized);
       BufferedInputStream bufIn = new BufferedInputStream(fileIn);
       ObjectInputStream objIn = new ObjectInputStream(bufIn);
	   JointClassifier initClassifier= (JointClassifier) objIn.readObject(); 
	   
       HashMultimap<String,String> catTextHash = HashMultimap.create();
       HashMap<String,String> titleMap = new HashMap<String,String>();
       
       int total = 0;
       int certas = 0;
       
       while((line = collection.readLine()) != null){
    	   String fields[] = line.split("\t");
    	   if(fields.length!= 2)
    		   continue;
    	   String text = fields[1];
    	   String title = fields[0];
    	   ConditionalClassification classification = initClassifier.classify(text);
    	   String bestCategory = classification.bestCategory();
    	   System.out.print(".");
    	   if(total % 100 == 0)
    		  System.out.println(total);
    	   
    	   catTextHash.put(bestCategory, title);
    	   total++;
    	   titleMap.put(fields[0], fields[1]);
       }
       System.out.println("total documents :" + total);
       initClassifier = null;
       int certas2=0;
       boolean exception = false;
       
       ArrayList<Double> medianArray = new ArrayList<Double>();
       BoundedPriorityQueue<ScoredObject<Pair<Double, Double>>> queue;
       MapSymbolTable mapCategories;
       ArrayList<Pair<Double,Double>> similarCoords;
       ArrayList<Double> similarScores;
       ScoredObject<Pair<Double, Double>> q;
       KnnNeighbours simClassifier = trainSimilarityModel();
       
       
       
       for(String cat : catTextHash.keySet()){
    	   try{
    	   fileIn = new FileInputStream(classifierDir+'/'+ cat + '_' + resolution + '_' + resInit + '_' + nGram + '_' + tokenized + "_train" );
    	   bufIn = new BufferedInputStream(fileIn);
	   	   objIn = new ObjectInputStream(bufIn);
	       mClassifier = (JointClassifier) objIn.readObject(); 
    	   }catch(IOException e){ exception = true; }
    	   System.out.println("For sub Category " + cat + " " + catTextHash.get(cat).size() + "test cases");
    	   for(String title : catTextHash.get(cat)){
    		   String text = titleMap.get(title);
    		   titleMap.remove(title);
    		   String bestCategory;
    		   System.out.print(".");
    		   if(num % 100 == 0)
    			   System.out.println(num);
    		   if(!exception){   			   
    			   ConditionalClassification newclassification = mClassifier.classify(text);            
    			   bestCategory = newclassification.bestCategory();
    		   }else {bestCategory = cat;}
        	   double lat1 = getCatLatitude(bestCategory);
        	   double lon1 = getCatLongitude(bestCategory);
        	   
        	   try{
        		   queue = simClassifier.NearesNeighbours(text, bestCategory); //QUEUE WITH MORE SIMILAR (EUCLIDEAN) DOCUMENTS 
        		   mapCategories = simClassifier.getmCategorySymbolTable();	//MAP SYMBOL WITH THE CATEGORIES
        		  
        		   if(queue.size() > 5)
        			   System.out.println("Bigger... " + queue.size());
        	 
        		   similarCoords = new ArrayList<Pair<Double,Double>>();
        		   similarScores = new ArrayList<Double>();
        		   
        	      
        		   

        		   q = queue.poll();
        		   int i = 0;
        		   while(q != null){
        			   if(i == 5)
        			   		break;	
        			   	
        			   		
        			   	similarCoords.add(q.getObject());
    		  			similarScores.add(q.score());
    		  			
    		  		q = queue.poll();
    		  		i++;
        		   }
        		   
        		   Pair<Double, Double> newCoords = adjustCoords(similarCoords, similarScores, new Pair( getCatLatitude(bestCategory), getCatLongitude(bestCategory)));
 
        		   similarCoords.removeAll(similarCoords);
        		   similarScores.removeAll(similarScores);
        		   similarCoords = null;
        		   similarScores = null;
        		   geoWritter.write(title + '\t' + newCoords.a() + '\t' + newCoords.b() + '\t' + text + '\n');
        		  }catch(NullPointerException n){geoWritter.write("" + getCatLatitude(bestCategory) + '\t' + getCatLongitude(bestCategory) + '\t' + text + '\n');
        	   }
    		   if((num % 50) == 0)
    			   geoWritter.flush();
    			       		   
    		   num++;
    		   
    	   }
    	   mClassifier = null;
    	   
       }
       System.out.println("certas certas2 total " + certas + ' ' + certas2 + ' ' + total);
       
       System.out.println("total number of tests :" + total);
       System.out.println("ratio inicial :" + ((double) certas / (double) total));
       System.out.println("precisao secundario:" + ((double)certas2/(double) total));
       
       geoWritter.close();
       return classifiedGeo;
   }
   
   void classifyText(String text) throws IOException, ClassNotFoundException{
	   
	   FileInputStream fileIn = new FileInputStream(classifierDir+"/initClassifier_" + resolution + '_' + nGram + '_' + tokenized);
	   BufferedInputStream bufIn = new BufferedInputStream(fileIn);
       ObjectInputStream objIn = new ObjectInputStream(bufIn);
	   JointClassifier initClassifier= (JointClassifier) objIn.readObject(); 
	   
	   ConditionalClassification classification = initClassifier.classify(text);
	   String initCat = classification.bestCategory();
	   fileIn = new FileInputStream(classifierDir+'/'+ initCat + '_' + resolution + '_' + resInit + '_' + nGram + '_' + tokenized + "_train" );
	   bufIn = new BufferedInputStream(fileIn);
   	   objIn = new ObjectInputStream(bufIn);
       mClassifier = (JointClassifier) objIn.readObject(); 
	   
       ConditionalClassification newclassification = mClassifier.classify(text);            
	   String bestCategory = newclassification.bestCategory();
	   double lat = getCatLatitude(bestCategory);
	   double lon = getCatLongitude(bestCategory);
	   System.out.println(lat + '\t' + lon);
	   
   }
   
   
   
   
   double median(ArrayList<Double> list){
	   
	   Collections.sort(list);
	   int num = list.size();
	   
	   if(num%2 != 0)
		   return list.get((num + 1) /2);
	   else
		   return (list.get(num/2) + list.get(num/2 + 1))/2;
		   
	   
	   
	   
   }
   
   
   
   
   
   /* 
    * evaluates a test data collection using four different methods to assingn coordinates
    * 
    * 
    */
   
    void evaluate() throws IOException, ClassNotFoundException {
        System.out.println("\nEvaluating.");
        int numTestsInit = 0;
        int numCorrectInit = 0;
        testTextHash = getTextHash(testData);
        double media = 0.0;
        int numMedia = 0;
        FileWriter results;  
        FileWriter graphics;
        HashMap<String, Integer> numCatHash = new HashMap<String, Integer>();
        FileInputStream fileIn = new FileInputStream(classifierDir+"/initClassifier_" + resolution + '_' + nGram + '_' + tokenized);
        
        File fileScores = new File("scores.txt");
        FileWriter scoreW = new FileWriter(fileScores);
        BufferedInputStream bufIn = new BufferedInputStream(fileIn);
        ObjectInputStream objIn = new ObjectInputStream(bufIn);
        mClassifier= (JointClassifier) objIn.readObject(); 
        String type;
        if(tokenized == 1)
        	type = "token";
        else
        	type = "chars";
        File file = new File("results" + resolution + "_" + multi + '_' + resInit + '_' + type +  ".txt" );
        results = new FileWriter (file);
        graphics = new FileWriter(new File("graphics_"+ resolution + '_' + resInit  + '_' + tokenized + ".txt"));
	    if(tokenized == 0)
        	graphics.write("title\trealLat\trealLon\tLat\tLon\tdistance4\tdistance\tdistance2\tdistance3\tdistance42\tdistance43\tdistance44\n");
        else graphics.write("title\trealLat\trealLon\tLat\tLon\tdistance\tdistance2\tdistance3\n");
        
        
        KnnNeighbours simClassifier = trainSimilarityModel();
        
        System.out.println("Numero de testes : " + testTextHash.size());
       
        for(String title : testTextHash.keySet()){
        
            String category = getCatInit( new Double(coorHash.get(title)[0]) , new Double(coorHash.get(title)[1]), spindexInit );          
            String text = title + '\t' + testTextHash.get(title);
            numTestsInit++;
            ConditionalClassification classification = mClassifier.classify(text);            
            String bestCategory = classification.bestCategory();
            System.out.println("classified: " + bestCategory + "categoria:" + category);

            if (bestCategory.equals(category))
            	numCorrectInit++;
            System.out.print('.');
            
           catTestClassifications.put(bestCategory, title);
			
        }
        System.out.println("numCorrect: "+ numCorrectInit + " numTests: " + numTestsInit);
        System.out.println("\n Average Correct: " + (double)numCorrectInit / (double)numTestsInit);
        results.write("\n Average Correct: " + (double)numCorrectInit / (double)numTestsInit +'\n');
        results.flush();
        
        
        //HIERARQUIA DOIS!
        
        
        int numTests = 0;
        int numCorrect = 0;
    	double errorDistance = 0;
    	double errorDistance4 = 0;
        double errorDistance2 = 0;
        double errorDistance3 = 0;     
        double errorDistance42 = 0;
        double errorDistance43 = 0;
        double errorDistance44 = 0;
        double errorRightDistance = 0;
        double errorRightDistance2 = 0;
        double errorRightDistance3 = 0;
        double errorRightDistance4 = 0.0;
        double errorWrongDistance = 0.0;
        double errorWrongDistance4 = 0.0;
        
        int numWrong = 0;
        List<Double> errorD = new ArrayList<Double>();
        List<Double> errorD2 = new ArrayList<Double>();
        List<Double> errorD3 = new ArrayList<Double>();
            

            
        ArrayList<Double> distances2 = new ArrayList<Double>();    
        ArrayList<Double> distances3 = new ArrayList<Double>();   
            
        for(int d = 2; d< 40; d++){
        	distances2.add(0.0);
        	distances3.add(0.0);    	
        }
        

        for(String cat : mCategoriesInit){
        	System.out.println("\n Para a categoria inicial " + cat);
            long init = System.currentTimeMillis();
            try{
            	fileIn = new FileInputStream(classifierDir+'/'+cat + '_' + resolution + '_' + resInit + '_' + nGram + '_' + tokenized + "_train" );
            }catch(IOException e){continue;}
            bufIn = new BufferedInputStream(fileIn);
            objIn = new ObjectInputStream(bufIn);
            mClassifier = (JointClassifier) objIn.readObject();  
            for(String title : catTestClassifications.get(cat)){
            	
            	numTests++;      	
            	JointClassification classification = mClassifier.classify(title + ' ' + testTextHash.get(title));            
                String bestCategory = classification.bestCategory();
                String category = getCat(new Double(coorHash.get(title)[0]), 
             	    		new Double(coorHash.get(title)[1]), spindex); 
                double distance = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
             	    		new Double(coorHash.get(title)[1]),
             						                     getCatLatitude(bestCategory),
               						                     getCatLongitude(bestCategory));
                errorDistance += distance;
             	
             	
             	for(int i= 0; i< classification.size(); i++){
             		if(classification.conditionalProbability(i) > 0.0){
             			
             			//System.out.println(" conditional :" + classification.conditionalProbability(i));
             			Double d = new Double(classification.conditionalProbability(i));
             			
             		}
             			if(i >0){
             				media += classification.conditionalProbability(i);
             				numMedia++;
             			}
             		
             	}
             	
	
             	    //DISTANCE 2
             	Map<String,Double> classes = new HashMap<String,Double>();
             	int aux = classification.size();
             	
             	double distance2 =0 ;
             	double distance3 = 0;
             	
             	


             		for ( int i = 0 ; i < aux ; i++ ) {
             			if(classification.conditionalProbability(i) == 0.0)
             				continue;
             			classes.put(classification.category(i), classification.conditionalProbability(i));
             		}
	
             		            		
             	
             	
             		distance2= VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), new Double(coorHash.get(title)[1]),
         				 		                      getLatitude(classes),
           						                      getLongitude(classes));
             	    
             		
             		//DISTANCE 3
             		classes = new HashMap<String,Double>();
             		classes.put(bestCategory,classification.conditionalProbability(0));
             		for ( String nCat : getNeighbourCats(bestCategory) ) 
             			try {
             				if(classification.conditionalProbability(catAdaptor(nCat)) == 0.0)
             					continue;
             				classes.put(nCat,classification.conditionalProbability(nCat));          	    	
             			}catch ( java.lang.IllegalArgumentException ex ) { continue; }
           	    
             		distance3 = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
             				new Double(coorHash.get(title)[1]),
           				 		                      getLatitude(classes),
             						                      getLongitude(classes));
             		
             	
             	errorDistance2 += distance2;
             	errorDistance3 += distance3;
             	errorD.add(distance);
             	errorD2.add(distance2);
             	errorD3.add(distance3);
             	   
             	
                double distance4 = 0.0;
				double distance42 = 0.0;
				double distance43 = 0.0;
				double distance44 = 0.0;
				
				
                if(tokenized == 0){
                 
					classes = new HashMap<String,Double>();
					
					BoundedPriorityQueue<ScoredObject<Pair<Double, Double>>> queue = simClassifier.NearesNeighbours(testTextHash.get(title), bestCategory); //QUEUE WITH MORE SIMILAR (EUCLIDEAN) DOCUMENTS 
					
					MapSymbolTable mapCategories = simClassifier.getmCategorySymbolTable();	//MAP SYMBOL WITH THE CATEGORIES
					ArrayList<Pair<Double,Double>> similarCoords = new ArrayList<Pair<Double,Double>>();
					ArrayList<Pair<Double,Double>> similarCoords2 = new ArrayList<Pair<Double,Double>>();
					ArrayList<Pair<Double,Double>> similarCoords3 = new ArrayList<Pair<Double,Double>>();
					ArrayList<Pair<Double,Double>> similarCoords4 = new ArrayList<Pair<Double,Double>>();
              	 
					ArrayList<Double> similarScores = new ArrayList<Double>();
					ArrayList<Double> similarScores2 = new ArrayList<Double>();
					ArrayList<Double> similarScores3 = new ArrayList<Double>();
					ArrayList<Double> similarScores4 = new ArrayList<Double>();
              	 
              	 
              	 	HashMap<Pair<Double, Double>,Double > similar = new HashMap<Pair<Double, Double>,Double >();
					ScoredObject<Pair<Double, Double>> q = queue.poll();
					scoreW.write("FOR document title : " + title + ' ' + bestCategory +" classified coords: "+ getCatLatitude(bestCategory) +' ' + getCatLongitude(bestCategory) + " real coords : "+ new Double(coorHash.get(title)[0]) + ' ' +  new Double(coorHash.get(title)[1]) + '\n' );
              	 	int number = 0;
        		  	while(q != null){
        		  		if(number >0 && number <5){
        		  			similarCoords.add(q.getObject());
        		  			similarScores.add(q.score());
        		  			
        		  			similarCoords2.add(q.getObject());
        		  			similarScores2.add(q.score());
        		  			
        		  			similarCoords3.add(q.getObject());
        		  			similarScores3.add(q.score());
        		  			
        		  			similarCoords4.add(q.getObject());
        		  			similarScores4.add(q.score());
        		  			//System.out.println("Score : " + q.score() + " coords :" + q.getObject() + " cat " + getCat(q.getObject().a(), q.getObject().b(), spindex));
        		  			scoreW.write("Score : " + q.score() + " coords :" + q.getObject() + " cat " + getCat(q.getObject().a(), q.getObject().b(), spindex) + '\n');
        		  		
        		  			
        		  		}
        		  		if(number >= 5 && number < 10){
        		  			
        		  			similarCoords2.add(q.getObject());
        		  			similarScores2.add(q.score());
        		  			
        		  			similarCoords3.add(q.getObject());
        		  			similarScores3.add(q.score());
        		  			
        		  			similarCoords4.add(q.getObject());
        		  			similarScores4.add(q.score());
        		  			
        		  			
        		  			
        		  		}
        		  		if(number >= 10 && number < 15){
        		  			
        		  			similarCoords3.add(q.getObject());
        		  			similarScores3.add(q.score());
        		  			
        		  			similarCoords4.add(q.getObject());
        		  			similarScores4.add(q.score());
        		  			
        		  			
        		  		}
        		  		if(number >= 15){
    		  			
        		  			similarCoords4.add(q.getObject());
        		  			similarScores4.add(q.score());
        		  			
        		  		}	
        		  		q = queue.poll();
        		  		number++;
        		  	}
        		  	
        		 	Pair<Double, Double> newCoords = adjustCoords(similarCoords, similarScores, new Pair( getCatLatitude(bestCategory), getCatLongitude(bestCategory)));
					scoreW.write("final coords : " + newCoords.a() + ' ' + newCoords.b() );
					scoreW.write("\n\n");
					scoreW.flush();
        		 
					distance4 = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
          				new Double(coorHash.get(title)[1]),
        				 		                      newCoords.a().doubleValue(),
          						                      newCoords.b().doubleValue());
					errorDistance4+= distance4;
        		 
					newCoords = adjustCoords(similarCoords2, similarScores2, new Pair( getCatLatitude(bestCategory), getCatLongitude(bestCategory)));
        		 
					distance42 = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
           				new Double(coorHash.get(title)[1]),
	                      newCoords.a().doubleValue(),
	                      newCoords.b().doubleValue());
					errorDistance42 += distance42;
        		 
        		 
					newCoords = adjustCoords(similarCoords3, similarScores3, new Pair( getCatLatitude(bestCategory), getCatLongitude(bestCategory)));
        		 
					distance43 = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
           				new Double(coorHash.get(title)[1]),
	                      newCoords.a().doubleValue(),
	                      newCoords.b().doubleValue());
					errorDistance43 += distance43;
        		 
					newCoords = adjustCoords(similarCoords4, similarScores4, new Pair( getCatLatitude(bestCategory), getCatLongitude(bestCategory)));
        		 
					distance44 = VincentyDistanceCalculator.getDistance(new Double(coorHash.get(title)[0]), 
           				new Double(coorHash.get(title)[1]),
	                      newCoords.a().doubleValue(),
	                      newCoords.b().doubleValue());
					errorDistance44 += distance44;
        		 
					boolean right = false;
	              	if (bestCategory.equals(category)){
	              		right = true;
	              		errorRightDistance += distance;
	              		errorRightDistance2 += distance2;
	              		errorRightDistance3 += distance3;
	              		errorRightDistance4 += distance4;
	              		++numCorrect;
	              	 }else {
	              		 errorWrongDistance += distance;
	              		 errorWrongDistance4 += distance4;
	              		 numWrong++;
	              		 
	              	 }
					
					
					
        		 graphics.write(title + '\t' +  coorHash.get(title)[0] + '\t' + coorHash.get(title)[1] + "\t" +
        				 newCoords.a().doubleValue() + '\t' + newCoords.b().doubleValue() + '\t' + distance4 + '\t'+ distance + '\t' + distance2 + '\t' + distance3 + '\t'+ '\t' +distance42 + '\t' +distance43 + '\t' + distance44 + '\t' + right + '\n');
        		 } else 
				 graphics.write(title + '\t' +  coorHash.get(title)[0] + '\t' + coorHash.get(title)[1] + "\t" +
        				 getCatLatitude(bestCategory) + '\t' + getCatLongitude(bestCategory) + '\t' + distance + '\t'+ distance2+ '\t' + distance3 + '\t' + bestCategory.equals(category) +'\n');
				 
               	
              	 if(numCatHash.get(bestCategory) == null)
             		 numCatHash.put(bestCategory , 0);
             	 else numCatHash.put(bestCategory, numCatHash.get(bestCategory) + 1);
             	 System.out.print('.');
        		 if(numTests % 50 == 0){
             		 System.out.println("tamanho: " + getArea(bestCategory));
             		 System.out.println(numTests);
             	     results.write("  Bins occupied :" + numCatHash.size() + '\n');
             	     results.write("  # Test Cases=" + numTests + '\n');
             	     results.write("  # Correct cells=" + numCorrect + '\n');
             	     results.write("  % Correct cells=" + ((double)numCorrect)/(double)numTests + '\n');
             	     results.write("  Average centroid Right distance=" + errorRightDistance/((double)numCorrect) + " Km\n");
             	     results.write("  Average centroid distance=" + errorDistance/((double)numTests) + " Km\n");
             	     results.write("  Standard deviation centroid distance " + deviation(errorD, errorDistance/((double)numTests)) + " Km\n");
             	     results.write("  Average centroid Right 2 distance=" + errorRightDistance2/((double)numCorrect) + " Km\n");
             	     results.write("  Average distance method 1=" + errorDistance2/((double)numTests) + " Km\n");
             	     results.write("  Standard deviation distance method 1 " + deviation(errorD2, errorDistance2/((double)numTests)) + " \n");
             	     results.write("  Average centroid Right distance3 =" + errorRightDistance3/((double)numCorrect) + " Km\n");
             	     results.write("  Average distance method 2=" + errorDistance3/((double)numTests) + " Km\n");
             	     results.write("  Standard deviation distance method 2 " + deviation(errorD3, errorDistance3/((double)numTests)) + " Km\n");
             	     results.write("  Average distance method 3=" + errorDistance4/((double)numTests) + " Km\n");
             	     results.write("  Average distance method 32=" + errorDistance42/((double)numTests) + " Km\n");
             	     results.write("  Average distance method 33=" + errorDistance43/((double)numTests) + " Km\n");
             	     results.write("  Average distance method 34=" + errorDistance44/((double)numTests) + " Km\n");
      				 results.write("Average Right distance method 3 = " + errorRightDistance4/(double)numCorrect +  " Km\n");
      				 results.write("Average Wrong distance method 3 = " + errorWrongDistance4/(double)numWrong +  " Km\n");
      				 results.write("Average Wrong distance method  = " + errorWrongDistance/(double)numWrong +  " Km\n");
           		 
        		 }
                 
                 results.flush();
                 
                }
                System.out.println("demorou: " + (System.currentTimeMillis() - init));

        	}

        
        System.out.println("  Bins occupied :" + numCatHash.size());
        System.out.println("  # Test Cases=" + numTests);
        System.out.println("  # Correct cells=" + numCorrect);
        System.out.println("  % Correct cells=" + ((double)numCorrect)/(double)numTests);        
        System.out.println("  Average centroid distance=" + errorDistance/((double)numTests) + " Km");
        System.out.println("  Standard deviation centroid distance " + deviation(errorD, errorDistance/((double)numTests)) + " Km");
        System.out.println("  Average distance method 1=" + errorDistance2/((double)numTests) + " Km");
        System.out.println("  Standard deviation distance method 1 " + deviation(errorD2, errorDistance2/((double)numTests)) + " Km");
        System.out.println("  Average distance method 2=" + errorDistance3/((double)numTests) + " Km");
        System.out.println("  Standard deviation distance method 2 " + deviation(errorD3, errorDistance3/((double)numTests)) + " Km");
        System.out.println("  Average distance method 3 5=" + errorDistance4/((double)numTests) + " Km");
        System.out.println("  Average distance method 3 10=" + errorDistance42/((double)numTests) + " Km");
        System.out.println("  Average distance method 3 15=" + errorDistance43/((double)numTests) + " Km");
        System.out.println("  Average distance method 3 20=" + errorDistance44/((double)numTests) + " Km");
        System.out.println("  Removidas " + removed + "categorias");
		results.write("\n --  FINAL  --\n");
        results.write("  Bins occupied :" + numCatHash.size()  + '\n');
        results.write("  # Test Cases=" + numTests + '\n');
        results.write("  # Correct cells=" + numCorrect + '\n');
        results.write("  % Correct cells=" + ((double)numCorrect)/(double)numTests + '\n');
        results.write("  Average centroid Right distance=" + errorRightDistance/((double)numCorrect) + " Km\n");
        results.write("  Average centroid distance=" + errorDistance/((double)numTests) + " Km\n");
        results.write("  Standard deviation centroid distance " + deviation(errorD, errorDistance/((double)numTests)) + " Km\n");
        results.write("  Average centroid Right 2 distance=" + errorRightDistance2/((double)numCorrect) + " Km\n");
        results.write("  Average distance method 1=" + errorDistance2/((double)numTests) + " Km\n");
        results.write("  Standard deviation distance method 1 " + deviation(errorD2, errorDistance2/((double)numTests)) + " \n");
        results.write("  Average centroid Right distance3 =" + errorRightDistance3/((double)numCorrect) + " Km\n");
        results.write("  Average distance method 2=" + errorDistance3/((double)numTests) + " Km\n");
        results.write("  Standard deviation distance method 2 " + deviation(errorD3, errorDistance3/((double)numTests)) + " Km\n");
        results.write("  Average distance method 3=" + errorDistance4/((double)numTests) + " Km\n");
        results.write("  Average distance method 32=" + errorDistance42/((double)numTests) + " Km\n");
  	    results.write("  Average distance method 33=" + errorDistance43/((double)numTests) + " Km\n");
  	    results.write("  Average distance method 34=" + errorDistance44/((double)numTests) + " Km\n");
		results.write("Average Right distance method 3 = " + errorRightDistance4/(double)numCorrect + " Km\n");
        scoreW.close();
        results.close();
        graphics.close();
        wordCount("test");    
        
    }
    
    
    public static double sigmoid ( double valor ) {
    	double constant = -27.85;
    	BigDecimal k = new BigDecimal(-1.0).subtract(new BigDecimal(Math.pow(10.0,constant)));
    	BigDecimal v2 = k.subtract(new BigDecimal(valor)).add(new BigDecimal(1.0));
    	BigDecimal v1 = k.multiply(new BigDecimal(valor));
    	return v1.divide(v2,1).doubleValue();
    }
    
    
    
    private static void report(SortedSet<ScoredObject<String[]>> nGrams, String category) throws IOException {
    	FileWriter writer = new FileWriter(new File("analysis_" + category + ".txt"));
        for (ScoredObject<String[]> nGram : nGrams) {
            double score = nGram.score();
            String[] toks = nGram.getObject();
            report_filter(score,toks, writer);
            writer.flush();
        }
    }
    
    private static void report_filter(double score, String[] toks, FileWriter writer) throws IOException{
    	
    	
        String accum = "";
        for (int j=0; j<toks.length; ++j) {
            accum += " "+toks[j];
        }
        writer.write("Score: "+score+" with :"+accum + '\n');
        System.out.println("Score: "+score+" with :"+accum);
    }
    
   
 
    
    
    public void runTrain() throws ClassNotFoundException, IOException{
    	
    	
    	
    	train();
    	train2();
    	analysisWriter.close();
    }
    
    public void runTest() throws ClassNotFoundException, IOException{

    	evaluate();
    	
    }

    
    
    public void analyse() throws IOException{
    	
    	wordCount("train");
    	wordCount("test");
    	
    }
    
    
    public static void main(String[] args) throws Exception {
    	
    	//COLOCAR O TIPO DE RUN PRIMEIRO!!
    	if(Integer.parseInt(args[7]) == 0)
    		new DocumentGeocoderLMNew(args).run();
    	else if(Integer.parseInt(args[7]) == 1)
        	new DocumentGeocoderLMNew(args).runTrain();
    	else if(Integer.parseInt(args[7]) == 2)
        	new DocumentGeocoderLMNew(args).runTest();
    	else if(Integer.parseInt(args[7]) == 3)
    		new DocumentGeocoderLMNew(args).classifyCollection(args[8]);
    	else if(Integer.parseInt(args[8]) == 4)
    		new DocumentGeocoderLMNew(args).classifyText(args[9]);
    }

}
